Project - 4 - Advanced Lane Lines¶
Due Date: January 24th, 2017
In [457]:
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os
from moviepy.editor import VideoFileClip
from moviepy.editor import CompositeVideoClip
from IPython.display import HTML
print('Imports successful.')
Calibration¶
In [458]:
# Load all the calibration images and convert to gray scale.
directory = "CarND-Advanced-Lane-Lines/camera_cal/"
imgfilenames = sorted(os.listdir(directory))
pixelh = 720
pixelw = 1280
cal_images_gray = np.zeros( [len(imgfilenames), pixelh, pixelw], dtype='uint8')
for num, name in enumerate(imgfilenames):
# Calibration images 07 and 15 have an extra height and width pixel for some reason.
if num == 6 or num == 14:
imgtemp = cv2.cvtColor(mpimg.imread(directory + name), cv2.COLOR_RGB2GRAY)
cal_images_gray[num] = imgtemp[1:721, 1:1281]
continue
cal_images_gray[num] = cv2.cvtColor(mpimg.imread(directory + name), cv2.COLOR_RGB2GRAY)
print(name)
In [459]:
# Find Chessboard corners.
objpoints = []
imgpoints = []
objp = np.zeros((9*6,3), np.float32)
#print(np.mgrid[0:8,0:6])
#print('\n')
#print(np.mgrid[0:8, 0:6].T.reshape(-1,2))
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)
#print(objp)
for i in range(20):
image = cal_images_gray[i]
ret, corners = cv2.findChessboardCorners(image, (9,6), None)
print(str(i) + ': '+ str(ret))
if ret == True:
imgpoints.append(corners)
objpoints.append(objp)
image = cv2.drawChessboardCorners(image, (9,6), corners, ret)
plt.imshow(image)
In [460]:
def img_undistort(img, objpoints, imgpoints):
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) # Must read image using mpimg.imread
output = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
undist = cv2.undistort(img, output[1], output[2], None, output[1])
return undist, output
disimage = mpimg.imread(directory + imgfilenames[18])
plt.imshow(disimage)
plt.show()
undistort, camOut = img_undistort(disimage, objpoints, imgpoints)
plt.imshow(undistort)
plt.show()
Undistort Test Images¶
In [461]:
directory = "CarND-Advanced-Lane-Lines/test_images/"
imgfilenames = sorted(os.listdir(directory))
undist_test = []
for imgname in imgfilenames:
f, (p1, p2) = plt.subplots(1, 2, figsize=(10, 5))
f.tight_layout()
disimage = mpimg.imread(directory + imgname)
p1.imshow(disimage)
p1.set_title('Original Image: ' + imgname)
undistort, camOut = img_undistort(disimage, objpoints, imgpoints)
undist_test.append(undistort)
p2.imshow(undistort)
p2.set_title('Undistorted Image')
plt.show()
undist_test = np.asarray(undist_test)
Assume all lane lines are either yellow or white. Pure white is RGB = (255, 255, 255) and pure yellow is RGB = (255,255,0). So really a high threshold on both the R AND the G should force the remaining pixels to either be yellow or white.
In [462]:
# Yellow and white color threshold.
img = undist_test[5]
#threshmin = 130
threshmax = 255
def red_thresh(img, threshmin, threshmax):
retval, binaryred = cv2.threshold(img[:,:,0], threshmin, threshmax, cv2.THRESH_BINARY)
return binaryred
for threshmin in range(90, 211, 20):
binaryred = red_thresh(img, threshmin, threshmax)
retval, binarygreen = cv2.threshold(img[:,:,1], threshmin, threshmax, cv2.THRESH_BINARY)
f, (p1,p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(binaryred, cmap='gray')
p1.set_title('Red Threshold Min: '+ str(threshmin))
p2.imshow(binarygreen, cmap='gray')
p2.set_title('Green Threshold')
plt.show()
Thresholding for yellow and white colors only seems to work on dark pavement. On the light pavement colors it doesn't really pick anything up until the very high thresholds which remove other information.
However using a low threshold (like 90 or less) just on the red threshold could be used as an AND with other transforms since it could potentially help on the dark pavement sections and essentially have no effect on the light pavement sections.
HLS Threshold¶
In [463]:
# HLS - Test image 4 and 5 seem to be the best images that have both the light and dark pavement.
# Test image 4
hlsimg = cv2.cvtColor(undist_test[3], cv2.COLOR_RGB2HLS)
f, ((p1,p2),(p3,p4)) = plt.subplots(2,2, figsize=(10,5))
f.tight_layout()
p1.imshow(undist_test[3])
p1.set_title('Original testimage 4')
p2.imshow(hlsimg[:,:,0],cmap='gray')
p2.set_title('Hue')
p3.imshow(hlsimg[:,:,1],cmap='gray')
p3.set_title('Lightness')
p4.imshow(hlsimg[:,:,2],cmap='gray')
p4.set_title('Saturation')
plt.show()
# Test image 5
hlsimg = cv2.cvtColor(undist_test[4], cv2.COLOR_RGB2HLS)
f, ((p1,p2),(p3,p4)) = plt.subplots(2,2, figsize=(10,5))
f.tight_layout()
p1.imshow(undist_test[3])
p1.set_title('Original testimage 5')
p2.imshow(hlsimg[:,:,0],cmap='gray')
p2.set_title('Hue')
p3.imshow(hlsimg[:,:,1],cmap='gray')
p3.set_title('Lightness')
p4.imshow(hlsimg[:,:,2],cmap='gray')
p4.set_title('Saturation')
plt.show()
Saturation channel seems to clearly be the best but still am not happy with the results on their own. Looking at the last photo in the bottom right you can see that on the white pavement section the right lane lines hardly shows up at all.
In [464]:
# RGB to HLS with threshold transform.
def hls_transform(img, threshmin=0, threshmax=255):
# Convert to HLS form RGB.
hlsimg = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
# Create binary image with a threshold just on the saturation channel.
ret, binary = cv2.threshold(hlsimg[:,:,2], threshmin, threshmax, cv2.THRESH_BINARY)
return binary
In [465]:
threshmin = 90
threshmax = 200
transimg = undist_test[3]
for threshmin in range(70,151, 10):
f, (p1, p2) = plt.subplots(1, 2, figsize=(10, 5))
f.tight_layout()
p1.imshow(transimg)
p1.set_title('Original Image Min:'+ str(threshmin))
transimgoutput = hls_transform(transimg, threshmin, threshmax)
p2.imshow(transimgoutput, cmap='gray')
p2.set_title('Transform Image')
plt.show()
For saturation 100 seems like a good threshold point.
Prespective Transform¶
In [466]:
# Only use images 1, 4, 5, 6 since the image two isn't from the same video as as the project
# and for some reason in image 3 the cars hood is edited out. This could mess up the test transform.
%matplotlib notebook
plt.figure(figsize=(10,5))
plt.imshow(undist_test[0])
plt.show()
In [467]:
%matplotlib inline
# Four coordinates in original image. (width, height).
srccord = np.float32(
[[600, 460], # Top left.
[720, 460], # Top right.
[265, 720], # Bottom left.
[1125, 720]])# Bottom right.
# Four coordinates in transformed image.
dstcord = np.float32(
[[280, 200], # Top left.
[1000, 200], # Top right.
[280, 720], # Bottom left.
[1000, 720]]) # Bottom right.
imgorig = np.copy(undist_test[0])
img = cv2.line(imgorig, tuple(srccord[0].reshape(1, -1)[0]),
tuple(srccord[1].reshape(1, -1)[0]), (255,255,255), thickness=4, lineType=8, shift=0)
img = cv2.line(img, tuple(srccord[2].reshape(1, -1)[0]),
tuple(srccord[3].reshape(1, -1)[0]), (255,255,255), thickness=4, lineType=8, shift=0)
img = cv2.line(img, tuple(srccord[0].reshape(1, -1)[0]),
tuple(srccord[2].reshape(1, -1)[0]), (255,255,255), thickness=4, lineType=8, shift=0)
img = cv2.line(img, tuple(srccord[3].reshape(1, -1)[0]),
tuple(srccord[1].reshape(1, -1)[0]), (255,255,255), thickness=4, lineType=8, shift=0)
plt.imshow(imgorig)
Out[467]:
No real math or special method was used to get a starting point for perspective transform. Just manually placing the points by inspecting the image seemed to work fine for everything except how far down the road it should look before the resolution of the camera fails to provide enough data. I also could not think of a solid way of using math to determine what the perspective points should be since there are no perfectly straight sections of the road. If there was a nice straight section of road that could be used in a real case to determine the ideal perspective transform points.
To determine how far along the road its possible to look I ran a number of different distances below and determined the point at which the camera resolution doesn't provide enough information.
In [468]:
for topcord in range(420,481,5):
srccord[0,1] = topcord
srccord[1,1] = topcord
M = cv2.getPerspectiveTransform(srccord,dstcord)
Minv = cv2.getPerspectiveTransform(dstcord, srccord)
warped = cv2.warpPerspective(undist_test[5], M, (1280,720), flags=cv2.INTER_LINEAR)
f = plt.figure(figsize=(10,5))
plt.imshow(warped)
f.suptitle('Top Cord: ' + str(topcord))
plt.show()
Based on the above plots its seems that before 465, there isn't enough lane line information in the image (due to the resolution of the camera). Before 465, even through my own inspection it is difficult to make out the lane lines.
In [469]:
def pers_transform(img, inv='no'):
# Four coordinates in original image. (width, height).
srccord = np.float32(
[[600, 465], # Top left.
[720, 465], # Top right.
[265, 720], # Bottom left.
[1125, 720]])# Bottom right.
# Four coordinates in transformed image.
dstcord = np.float32(
[[280, 200], # Top left.
[1000, 200], # Top right.
[280, 720], # Bottom left.
[1000, 720]]) # Bottom right.
if inv == 'no':
M = cv2.getPerspectiveTransform(srccord,dstcord)
if inv == 'yes':
M = cv2.getPerspectiveTransform(dstcord, srccord)
pimg = cv2.warpPerspective(img, M, (1280,720), flags=cv2.INTER_LINEAR)
return pimg
for i in [0,3,4,5]:
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
warped = pers_transform(undist_test[i])
p1.imshow(warped)
p2.imshow(undist_test[i])
plt.show()
Gradient Threshold¶
In [470]:
%matplotlib inline
# X or Y Threshold
def abs_threshold(img, orient, threshmin, threshmax):
# Convert to grayscale.
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Take derivative in x or y given orient.
if orient == 'x':
sobel = cv2.Sobel(gray, cv2.CV_64F, 1,0)
if orient == 'y':
sobel = cv2.Sobel(gray, cv2.CV_64F, 0,1)
# Take absolute value of derivative.
sobel_abs = np.absolute(sobel)
# Scale to 0 - 255 and convert to unsigned interger.
scaled_sobel = np.uint8(255*sobel_abs/np.max(sobel_abs))
# Create binary output (1 indicates threshold is met).
binary_output = np.zeros_like(scaled_sobel)
binary_output[(scaled_sobel >= threshmin) & (scaled_sobel <= threshmax)] = 1
return binary_output
# Magnitude Gradient
def mag_threshold(img, sobel_kernel, threshmin, threshmax):
# Convert to grayscale.
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate gradient in both x and y direction.
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1,0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0,1, ksize=sobel_kernel)
# Calculate the magnitude of the gradient.
mag = np.sqrt(sobelx**2 + sobely**2)
# Rescale to 0 - 255 and convert to unsigned interger.
scale_factor = np.max(mag)/255
mag = (mag/scale_factor).astype(np.uint8)
# Create binary output (1 indicates threshold is met).
binary_output = np.zeros_like(mag)
binary_output[(mag >= threshmin) & (mag <= threshmax)] = 1
return binary_output
# Direction Gradient
def dir_threshold(img, sobel_kernel, threshmin, threshmax):
# Convert to grayscale.
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate gradient in both x and y direction.
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1,0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0,1, ksize=sobel_kernel)
# Suppress error messages.
with np.errstate(divide='ignore', invalid='ignore'):
# Find the direction of the gradient.
dgrad_abs = np.absolute(np.arctan(sobely/sobelx))
# Create binary output (1 indicates threshold is met).
binary_output = np.zeros_like(dgrad_abs)
binary_output[(dgrad_abs >= threshmin) & (dgrad_abs <= threshmax)] = 1
return binary_output.astype(np.uint8)
In [471]:
# Run all sobel gradients to compare and analyze possible combination of thresholds.
def total_sobel(img, xthresh, ythresh, magkernel, magthresh, dirkernel, dirthresh):
# Original Image
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(img)
p1.set_title('Original Image',color='w')
p2.imshow(pers_transform(img))
p2.set_title('Prespective Transform')
plt.show()
# X Gradient
orient = 'x'
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
ximg = abs_threshold(img, orient, xthresh[0], xthresh[1])
p1.imshow(ximg, cmap='gray')
p1.set_title('X Gradient',color='w')
p2.imshow(pers_transform(ximg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
# Y Gradient
orient = 'y'
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
yimg = abs_threshold(img, orient, ythresh[0], ythresh[1])
p1.imshow(yimg, cmap='gray')
p1.set_title('Y Gradient',color='w')
p2.imshow(pers_transform(yimg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
# Magnitude Gradient
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
magimg = mag_threshold(img, magkernel, magthresh[0], magthresh[1])
p1.imshow(magimg, cmap='gray')
p1.set_title('Magnitude Gradient',color='w')
p2.imshow(pers_transform(magimg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
# Direction Gradient
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
dirimg = dir_threshold(img, dirkernel, dirthresh[0], dirthresh[1])
p1.imshow(dirimg, cmap='gray')
p1.set_title('Direction Gradient', color='w')
p2.imshow(pers_transform(dirimg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
A final gradient output that is a combination of mutliple thresholds (absolute, magnitude, direction) is required to help find the lane lines in an image. While the output doesn't have to be perfect, because I will end up combining what I find with the HLS color threshold, it does have to be fairly good and seeing the lane lines.
Once I tune the parameters for a very good gradient that works on all of the images (1,4,5,6) I'll then fine tune the parameters so it can work better when I combine it with the HLS threshold and the R color threshold.
In [472]:
xthresh = [30, 150]
ythresh = [40, 100]
magkernel = 7
magthresh = [30, 100]
dirkernel = 7
dirthresh = [0.4, 1.1]
total_sobel(undist_test[5], xthresh, ythresh, magkernel, magthresh, dirkernel, dirthresh)
No amount of fine tuning seems to have the y gradient yield any information that is not in the x gradient. This would negate any benefit from using a y gradient in an OR with the x gradient. Also the y gradient is far too noisy for things outside the lane lines to be any use as a standalone addition. So I'm not going to use the y gradient for the final gradient combination.
The magnitude direction gradient on its own is very good at finding the lane lines however, it also pickups a great deal of noise. My main goal when selecting a direction gradient is to have it pick up the lane lines, but not the pieces of noise that can be seen in the magnitude gradient.
In the above picture it can be seen that the direction gardient picks up almost everything in the image, except for the noise seen in pixels 0-100 in the Y axis of the magnitude gradient. This means that using the direction gradient in an AND operator with the magnitude gradient should help filter out most of its noise.
In [473]:
def combined_grad(img, xthresh, magkernel, magthresh, dirkernel, dirthresh):
# Take absolute threshold in x direction.
abs_bin = abs_threshold(img, 'x', xthresh[0], xthresh[1])
mag_bin = mag_threshold(img, magkernel, magthresh[0], magthresh[1])
dir_bin = dir_threshold(img, dirkernel, dirthresh[0], dirthresh[1])
com_bin = np.zeros_like(dir_bin)
com_bin[ (abs_bin == 1) | ((mag_bin == 1) & (dir_bin == 1))] = 1
return com_bin
In [474]:
# Combined Gradient
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
combinedimg = combined_grad(undist_test[5], xthresh, magkernel, magthresh, dirkernel, dirthresh)
p1.imshow(undist_test[5])
p1.set_title('Dark Road', color='w')
p2.imshow(undist_test[3])
p2.set_title('Light Road', color='w')
plt.show()
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
combinedimg = combined_grad(undist_test[5], xthresh, magkernel, magthresh, dirkernel, dirthresh)
p1.imshow(combinedimg, cmap='gray')
p1.set_title('Combined Gradient - Dark Road', color='w')
p2.imshow(pers_transform(combinedimg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
combinedimg = combined_grad(undist_test[3], xthresh, magkernel, magthresh, dirkernel, dirthresh)
p1.imshow(combinedimg, cmap='gray')
p1.set_title('Combined Gradient - Light Road', color='w')
p2.imshow(pers_transform(combinedimg), cmap='gray')
p2.set_title('Prespective Transform')
plt.show()
The combined gradient performs well on dark road surfaces. However even the combined gradient performs poorly on light road surfaces. This is due to the lack of difference between a light road surface and a light painted line. In order to capture the lane line data on a light road surface the color or HLS thresholds will be used.
Combining Color, HLS, and Gradient¶
The first threshold I performed only on the red color channel I believe is still beneficial. A very low threshold (easily passed) will be used that will essentially filter out any object that doesn't have a strong red component (which the white and yellow lines will have plenty of). Thus I will use the red color channel threshold as an AND with both the HLS and gradient.
The HLS and gradient aspects will be used in an OR operator since they both have different strengths. The gradient is really good at dark road surfaces (where there is a large difference between the light lines and the dark road) and the HLS is really good at picking up any deep colors, regardless of the light or darkness of the road surface. However inorder to avoid most of the noise the HLS threshold has to be set quite aggressive, which is why the gardient threshold is needed.
In [475]:
# The default values for this function contain the final fine tuned parameters.
def combined_total (img, ident='all', rthresh=[70,255], sthresh=[120,200], xthresh=[30,150],
magkernel=7, magthresh=[30,100], dirkernel=7, dirthresh=[0.4,1.23]):
# Calculate binary image using red threshold.
bin_red = red_thresh(img, rthresh[0], rthresh[1])
# Calculate binary image using saturation threshold.
bin_sat = hls_transform(img, sthresh[0], sthresh[1])
# Calculate binary image using gradient threshold.
bin_grd = combined_grad(img, xthresh, magkernel, magthresh, dirkernel, dirthresh)
# Combine binary images as:
# (red AND sat) OR (red AND grd).
total_bin = np.zeros_like(bin_grd)
total_bin[ ((bin_red >= 1) & (bin_sat >= 1)) | ((bin_red >= 1) & (bin_grd >= 1)) ] = 1
if ident == 'all':
return total_bin
if ident == 'red':
return bin_red
if ident == 'sat':
return bin_sat
if ident == 'grd':
return bin_grd
In [476]:
for i in [0, 3,4,5]:
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(pers_transform(undist_test[i]))
p1.set_title('Test Image - '+str(i+1), color='w')
p2.imshow(pers_transform(combined_total(undist_test[i],'all')), cmap='gray')
p2.set_title('Combined Binary', color='w')
plt.show()
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(pers_transform(combined_total(undist_test[i],'sat')), cmap='gray')
p1.set_title('Saturation Threshold', color='w')
p2.imshow(pers_transform(combined_total(undist_test[i],'grd')), cmap='gray')
p2.set_title('Combined Gradient', color='w')
plt.show()
The final binary image parameters can be seen on 4 of the test images above. On all 4 images the lane lines are clearly shown. Looking at test image 5, my decision to include the red threshold as an AND operator on both saturation and gradient really pays off. As seen in the saturation threshold for test image 5, there is a great deal of noise on the Y axis around 300 pixels which isn't included in the final combined binary image. This is because the very weak red threshold (which allows almost everything in the image) clearly doesn't allow this dark shadow to pass through since it contains a very low red channel number.
Also using both gradient and saturation can be seen as necessary since there are some images where the saturation threshold is contributing almost all lane line information and there are other images where the majority of the lane line information is caputred by the gradient.
Estimate Lane Line Pixels¶
In [477]:
def find_lane_lines(img_in, chunk=4, start_pointl=[0], start_pointr=[0], output='img'):
# Create binary image containing lane lines and the rest of image.
img = pers_transform(combined_total(img_in,'all'))
lpeak = []
rpeak = []
# If no previous start point data, search for lines using sliding window.
if ((start_pointr[0] == 0) & (start_pointl[0] == 0)):
# Split the image into separate vertical chunks and find lane line location in each chunk.
for i in range(chunk-1,-1,-1):
# Generate histogram to find the lane line data.
histogram = np.sum(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),:], axis=0)
# Find area in historgram with most pixels which determines the right and left lanes.
lpeak.append(np.argmax(histogram[:750]))
rpeak.append(np.argmax(histogram[750:]))
# Code below is purely for visuals for histogram plot.
#histogram[lpeak[-1]-50] = 200
#histogram[lpeak[-1]+50] = 200
#histogram[rpeak[-1]-50+750] = 200
#histogram[rpeak[-1]+50+750] = 200
#plt.plot(histogram)
# If a starting point is provided, set r and l peak to it.
if start_pointr[0] != 0:
rpeak = start_pointr
if start_pointl[0] != 0:
lpeak = start_pointl
# Based on start point, see which direction lane line is moving by comparing
# number of pixels to the left and right of middle position.
for i in range(chunk-1, -1, -1):
# Count all pixels to the left of the previous frames window.
shiftl = np.count_nonzero(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
lpeak[chunk-1-i]-80:lpeak[chunk-1-i]])
# Count all pixels to the right of the previous frames window.
shiftr = np.count_nonzero(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
lpeak[chunk-1-i]:lpeak[chunk-1-i]+80])
# Used for debugging which direction window should move.
if i == 0:
shiftll = shiftl
shiftrr = shiftr
# If the difference between the left or right pixels is greater than 200, move window
# in that direction. If less than 200 pixels difference, stay where it is.
if (shiftl - shiftr) > 200:
lpeak[chunk-1-i] = lpeak[chunk-1-i] - 5
elif (shiftr - shiftl) > 200:
lpeak[chunk-1-i] = lpeak[chunk-1-i] + 5
# If line data lost, use histogram to re-acquire but must be within 150 pixels of last
# point where they were detected.
elif ((shiftr + shiftl) == 0):
histogram = np.sum(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
:], axis=0)
if (abs(lpeak[chunk-1-i] - np.argmax(histogram[:750])) < 150):
lpeak[chunk-1-i] = np.argmax(histogram[:750])
# Repeat above for right lane.
shiftl = np.count_nonzero(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
rpeak[chunk-1-i]-80+750:rpeak[chunk-1-i]+750])
shiftr = np.count_nonzero(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
rpeak[chunk-1-i]+750:rpeak[chunk-1-i]+80+750])
if (shiftl - shiftr) > 200:
rpeak[chunk-1-i] = rpeak[chunk-1-i] - 5
elif (shiftr - shiftl) > 200:
rpeak[chunk-1-i] = rpeak[chunk-1-i] + 5
elif ((shiftr + shiftl) == 0):
histogram = np.sum(img[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
:], axis=0)
if (abs(rpeak[chunk-1-i] - np.argmax(histogram[750:])) < 150):
rpeak[chunk-1-i] = np.argmax(histogram[750:])
# Create binary image which shows area of both left and right lane line.
peaks = np.zeros_like(img)
for i in range(chunk-1, -1, -1):
# This prevents error when trying to write values out of bounds of peaks.
if lpeak[chunk-1-i]-80 < 0:
lpeak[chunk-1-i] = 80
if rpeak[chunk-1-i]-80 < 0:
rpeak[chunk-1-i] = 80
# 160 pixel wide area where the peak of the histogram was found for each chunk.
peaks[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
lpeak[chunk-1-i]-80:lpeak[chunk-1-i]+80] = 1
peaks[int(img.shape[0]*i/chunk):int(img.shape[0]*(i+1)/chunk),
rpeak[chunk-1-i]-80+750:rpeak[chunk-1-i]+80+750] = 2
# If transformed image data overlaps with left and right lane line area, keep it.
lane_lines = np.zeros_like(img)
# Left lane lines are indicated by 1, right lane lines are 2. Everything else is 0.
# Commented out lines are used for debugging and showing windows and pixels.
#lane_lines[peaks == 1] = 2
lane_lines[ (img == 1) & (peaks == 1)] = 1
#lane_lines[peaks ==2] = 1
lane_lines[ (img == 1) & (peaks == 2)] = 2
# The loc output will be used for seeking the lane line starting position.
if output == 'img':
return lane_lines
if output == 'loc':
return lane_lines, lpeak, rpeak, shiftll, shiftrr
if output == 'peak':
return peaks
if output == 'peakloc':
return peaks, lpeak, rpeak
Show how previous iteration positional knowledge can be used for current lane estimation.¶
In [478]:
f, (p1, p2, p3) = plt.subplots(1,3, figsize=(10,5))
f.tight_layout()
p1.imshow(pers_transform(combined_total(undist_test[3],'all')),cmap='gray')
p1.set_title('Combined Threshold')
p2.imshow(find_lane_lines(undist_test[3],4, [0], [0], 'peak'), cmap='gray')
p2.set_title('Histogram Margin')
p3.imshow(find_lane_lines(undist_test[3],4, [0], [0], 'img'), cmap='gray')
p3.set_title('Final')
plt.show()
In [479]:
# Show ignoring current lane estimation due to being to far from starting point.
start_pointl = [240, 217, 198, 195, 194, 200]
start_pointr = [300, 310, 300, 330, 345, 330]
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(find_lane_lines(undist_test[3],6, start_pointl, start_pointr, 'peak'),cmap='gray')
p1.set_title('With Starting Point')
p2.imshow(find_lane_lines(undist_test[3],6, [0], [0], 'peak'), cmap='gray')
p2.set_title('Without Starting Point')
plt.show()
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(find_lane_lines(undist_test[3],6, start_pointl, start_pointr),cmap='gray')
p1.set_title('With Starting Point')
p2.imshow(find_lane_lines(undist_test[3],6), cmap='gray')
p2.set_title('Without Starting Point')
plt.show()
In [480]:
for i in [0,4,5]:
f, (p1, p2) = plt.subplots(1,2, figsize=(10,5))
f.tight_layout()
p1.imshow(pers_transform(combined_total(undist_test[i])),cmap='gray')
p2.imshow(find_lane_lines(undist_test[i],4), cmap='gray')
plt.show()
Measuring Radius of Curvature¶
In [481]:
# Input requires binary image with only estiamted lane line pixels.
def rad_curve(bin_img):
# Change image data into row (y axis) and column (x axis) information.
locl = np.where(bin_img == 1)
locr = np.where(bin_img == 2)
# Fit each lane line to a 2nd order polynomial.
lfit = np.polyfit(locl[0], locl[1], 2)
rfit = np.polyfit(locr[0], locr[1], 2)
# Generate polynomial line data.
yval = np.linspace(0,100, num=101)*7.2
rfitx = rfit[0]*yval**2 + rfit[1]*yval + rfit[2]
lfitx = lfit[0]*yval**2 + lfit[1]*yval + lfit[2]
# Plots useful for visualizing data.
#plt.plot(locl[1], locl[0],'x')
#plt.plot(locr[1], locr[0],'x', color='r')
#plt.plot(rfitx, locr[0], color='g', linewidth=2)
#plt.plot(lfitx, locl[0], color='g', linewidth=2)
#plt.plot(afitx, locl[0], color='black')
#plt.gca().invert_yaxis()
# Assume lane width is 3.7 meters and dashed lane lines are 3 meters.
# Right lane dashed line goes from roughly pixel 500 to 600.
ydist_pix = 3/100 # 3 meters per 100 pixels.
# Left left lane to right lane is roughly 800 pixels.
xdist_pix = 3.7/800 # 3.7 meters per 800 pixels.
# Fit pixel to distance values to a 2nd order polynomial.
lfitd = np.polyfit(locl[0]*ydist_pix, locl[1]*xdist_pix, 2)
rfitd = np.polyfit(locr[0]*ydist_pix, locr[1]*xdist_pix, 2)
# Calculate radius of curvature at median (roughly middle) from the car.
y_carl = np.median(locl[0])
lrad = (( 1 + (2*lfitd[0]*y_carl + lfitd[1])**2 )**1.5) / np.absolute(2*lfitd[0])
y_carr = np.median(locr[0])
rrad = (( 1 + (2*rfitd[0]*y_carr + rfitd[1])**2 )**1.5) / np.absolute(2*rfitd[0])
# Calculate average between two radii of curvature.
avgrad = (lrad + rrad)/2
# Calculate distance from center. From images center of hood seems to be center of image
# which is at pixel 640.
# Find location of lane closest to the car based on polynomial.
y_carr = np.max(locr[0])
y_carl = np.max(locl[0])
llane = lfit[0]*y_carl**2 + lfit[1]*y_carl + lfit[2]
rlane = rfit[0]*y_carr**2 + rfit[1]*y_carr + rfit[2]
# Average two lane positions to find where center of the lane is.
midlane = (llane + rlane)/2
# Difference between middle of lane and center of car. Convert to m from pixels.
diff_center = (midlane-640)*xdist_pix # Negative is car is left of middle of lane.
return [lrad, rrad, avgrad, diff_center], [lfitx, rfitx, yval]
Put Information Back On Original Image¶
In [482]:
def overlay_curve(img, radinfo, unwarp='yes'):
overlay = np.zeros_like(img).astype(np.uint8)
pts_left = np.array([np.transpose(np.vstack([radinfo[1][0], radinfo[1][2]]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([radinfo[1][1], radinfo[1][2]])))])
pts = np.hstack((pts_left,pts_right))
# Draw onto lane.
cv2.fillPoly(overlay, np.int_([pts]), (0,255,0))
if unwarp == 'no':
return overlay
# Inverse perspective transform.
unwarp_layer = pers_transform(overlay, 'yes')
return cv2.addWeighted(img, 1, unwarp_layer,0.3, 0)
Video Generation¶
Now that I can find the lane lines and measure the curvature and distance from center, it is time to adapt what I have made to work on video files as opposed to single images.
In [454]:
class Line():
def __init__(self):
self.peak = []
self.count = 0
self.fitx = np.zeros((5, 101))
self.diff_c = np.zeros(5)
self.avgrad = np.zeros(5)
Left = Line()
Right = Line()
font = cv2.FONT_HERSHEY_SIMPLEX
def process_image(image):
# Used for only processing trouble section of video for debugging.
# Left.count = Left.count + 1
# if ((Left.count < 475) | (Left.count > 675 )):
# return image
if len(Left.peak) == 0:
undist = cv2.undistort(image, camOut[1], camOut[2], None, camOut[1])
#bin_img = pers_transform(combined_total(image,'all'))
bin_img, l, r, shiftl, shiftr = find_lane_lines(undist, 4, [0], [0], 'loc')
radinfo = rad_curve(bin_img)
final = overlay_curve(undist, radinfo, 'yes')
else:
undist = cv2.undistort(image, camOut[1], camOut[2], None, camOut[1])
bin_img, l, r, shiftl, shiftr = find_lane_lines(undist, 4, Left.peak, Right.peak, 'loc')
radinfo = rad_curve(bin_img)
# Shift all old fitx values down one.
for i in range(0,4):
Left.fitx[i] = Left.fitx[i+1]
Right.fitx[i] = Right.fitx[i+1]
Left.diff_c[i] = Left.diff_c[i+1]
Left.avgrad[i] = Left.avgrad[i+1]
# Replace
Left.fitx[-1] = radinfo[1][0]
Right.fitx[-1] = radinfo[1][1]
Left.diff_c[-1] = radinfo[0][3]
Left.avgrad[-1] = radinfo[0][2]
if (np.count_nonzero(Left.fitx) == 505):
# Update radinfo with averaged values.
radinfo[1][0] = np.average(Left.fitx,0)
radinfo[1][1] = np.average(Right.fitx,0)
radinfo[0][3] = np.average(Left.diff_c)
radinfo[0][2] = np.average(Left.avgrad)
# Display information on image.
final = overlay_curve(undist, radinfo, 'yes')
cv2.putText(final,'Radius of Curvature: '+str(int(radinfo[0][2]))+' m',(10,50),
font, 1,(255,255,255),2)
if radinfo[0][3] < 0:
cv2.putText(final,'Car is ' + str(abs(round(radinfo[0][3],2))) + ' m left of center.',(10,100),
font, 1,(255,255,255),2)
if radinfo[0][3] >= 0:
cv2.putText(final,'Car is '+str(round(radinfo[0][3],2))+' m right of center.',(10,100),
font, 1,(255,255,255),2)
else:
final = image
cv2.putText(final,'Radius of Curvature: Acquiring...',(10,50),
font, 1,(255,255,255),2)
# color_bin = np.dstack((bin_img*100, bin_img*100, bin_img*100))
# font = cv2.FONT_HERSHEY_SIMPLEX
# cv2.putText(color_bin,'Final,(10,650),
# font, 1,(255,255,255),2)
Left.peak = l
Right.peak = r
return final
vid_output = 'rad-test2-opt2.mp4'
clip1 = VideoFileClip("project_video.mp4")
vid_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time vid_clip.write_videofile(vid_output, audio=False)
In [ ]:
clip1 = VideoFileClip("bin-opt.mp4")
clip2 = VideoFileClip("bin-opt-raw.mp4")
clip3 = clip2.resize(0.25)
clip4 = VideoFileClip('peaks-opt.mp4')
clip5 = clip4.resize(0.25)
video = CompositeVideoClip([clip1,
clip3.set_pos((480,360)),
clip5.set_pos((480,180))])
%time video.write_videofile('composite.mp4', audio=False)
In [456]:
clip1 = VideoFileClip("total2-opt2.mp4")
clip2 = VideoFileClip("raw.mp4")
clip3 = clip2.resize(0.25)
clip4 = VideoFileClip('peak-opt2.mp4')
clip5 = clip4.resize(0.25)
clip6 = VideoFileClip('bin-opt2.mp4')
clip7 = clip6.resize(0.25)
video = CompositeVideoClip([clip1,
clip3.set_pos((640,0)),
clip5.set_pos((960,0)),
clip7.set_pos((960,180))])
%time video.write_videofile('total-pipeline.mp4', audio=False)
In [455]:
# Test
bin_img, l, r, shiftl, shiftr = find_lane_lines(undist_test[5],4, [0], [0],'loc')
radinfo = rad_curve(bin_img)
print(radinfo[1][0])
fitx = radinfo[1][0]
#print(len(undist_test[4]))
layer = overlay_curve(undist_test[5], radinfo)
plt.imshow(layer)
Out[455]:
In [439]:
# Test area.
class Line():
def __init__(self):
self.peak = []
self.count = 0
self.fitx = np.zeros((10, 101))
Left = Line()
Left.fitx[1] = radinfo[1][0]
print(Left.fitx)
print('\n\n')
def proc(prev_val,r):
for i in range(0,9):
Left.fitx[i] = Left.fitx[i+1]
print(i)
Left.fitx[-1,0] = 400
print(Left.fitx)
print(np.count_nonzero(Left.fitx))
proc(0,0)
l = np.average(Left.fitx,0)
print(l)